package gcontent.channel.http;

import java.io.IOException;
import java.io.InputStream;

import java.net.URL;
import java.util.ArrayList;
import java.util.Date;

import org.apache.http.Header;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpRequestBase;

import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.log4j.Logger;

import gcontent.channel.Channel;
import gcontent.channel.error.ChannelEx;
import gcontent.channel.http.ssl.MySSLSocketFactory;
import gcontent.entities.Content;
import gcontent.entities.Site;
import gcontent.entities.content.ContentManager;
import gcontent.entities.content.error.ContentEx;
import gcontent.entities.site.SiteURL;
import gcontent.record.error.RecordEx;
import gcontent.utils.DateUtil;
import gcontent.utils.URLUtils;
import gcontent.utils.Utils;

import gcontent.config.Configure;

public class ChannelHTTP extends Channel {
	
	public static final String TYPE_HTTP = "HTTP";
	public static final String TYPE_HTTPS = "HTTPS";
	
	/* default configuration values */
	public static final String DEF_USER_AGENT = "Mozilla/5.0 (Windows; U; Windows NT 5.1; es-ES; rv:1.9.1.3) Gecko/20090824 Firefox/3.5.3";
	public static final String DEF_ACCEPT_ENC = "gzip, deflate";
	public static final String DEF_SESSION = "Guantanamera";
	
	public static final int DEF_MAX_CONTENT_LEN = 10485760; /* 10 megabytes */
	public static final int DEF_MAX_REDIRECTS = 5; /* 5 redirects */
	
	/* configuration parameters */
	public static final String CONFIG_USER_AGENT = "HTTP_USER_AGENT";
	public static final String CONFIG_ACCEPT_ENC  = "HTTP_ACCEPT_ENC";
	public static final String CONFIG_SESION  = "HTTP_SESION";
	public static final String CONFIG_MAX_CONTENT_LEN  = "MAX_CONTENT_LEN";

	private CookieManager cookie_mgr = new CookieManager();
	
	/* http connection */
	private HttpClient http_c = new DefaultHttpClient();
	private HttpRequestBase http_req = null;
	private HttpResponse http_resp = null;
	private int status_code = -1;

	/* bound data size to download */
	private int max_content_len = DEF_MAX_CONTENT_LEN;
	private int max_redirects = DEF_MAX_REDIRECTS;
	
	private static Logger log = Logger.getLogger(ChannelHTTP.class);
		
	public ChannelHTTP()
	{	
		super();
	}

	public String getProtocol()
	{
		return TYPE_HTTP;
	}
	
	/* set global parameters */
	private void initHttpClient()
	{	
		this.http_c = new DefaultHttpClient();
		
		if(Configure.get("CONFIG_MAX_CONTENT_LEN") != null)
			this.max_content_len = Integer.parseInt(Configure.get("CONFIG_MAX_CONTENT_LEN")); 	
		
		
		if(Configure.get("CONFIG_HTTP_PROXY_HOST") != null) {
			HttpHost proxy;
			int port;
			
			if(Configure.get("CONFIG_HTTP_PROXY_PORT") != null)
				port = Integer.parseInt(Configure.get("CONFIG_HTTP_PROXY_PORT"));
			else
				port = 80;
			
			proxy = new HttpHost(Configure.get("CONFIG_HTTP_PROXY_HOST"), port);
			this.http_c.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
			this.http_c.getParams().setParameter("http.protocol.handle-redirects", false);
		}
				
	}
	
	/* set properties in http request */
	private void prepareRequest(Content content_cached, SiteURL site)
	{	
		String cookie_value;
		
		this.http_req = new HttpGet(site.getURL().toString());		

		if(Configure.get(CONFIG_USER_AGENT) != null)
			this.http_req.setHeader("User-Agent", Configure.get(CONFIG_USER_AGENT));
		else
			this.http_req.setHeader("User-Agent", DEF_USER_AGENT);
		
		if(Configure.get(CONFIG_ACCEPT_ENC) != null)
			this.http_req.setHeader("Accept-Encoding", Configure.get(CONFIG_ACCEPT_ENC));
		else
			this.http_req.setHeader("Accept-Encoding", DEF_ACCEPT_ENC);
		
		/* set modification date */
		if(content_cached != null && content_cached.getLastModified() != null)
			this.http_req.setHeader("If-Modified-Since", DateUtil.formatDate(content_cached.getLastModified()));

		
		/* set cookie value for this request */
		cookie_value = this.cookie_mgr.getCookieValues(site.getURL());
		if(cookie_value != null)
			this.http_req.setHeader("Cookie", cookie_value);
	
	}
	
	public String getUserAgent()
	{
		return this.http_req.getFirstHeader("User-Agent") != null? this.http_req.getFirstHeader("User-Agent").getValue() : "";
	}

	private HttpResponse getResponse(Content content_cached, SiteURL site) throws IOException, ChannelEx
	{
		/* initialize http_client */
		initHttpClient();
		
		/* prepare request headers */
		prepareRequest(content_cached, site);	
		
		/* register our ssl socket factory */
		try { 
			MySSLSocketFactory.register(this.http_c, site.getURL());
		} catch(Exception ex) {
			throw new ChannelEx(ex);
		}

		/* return response */
		return this.http_c.execute(this.http_req);
	}
	
	
	private byte[] readContent(HttpResponse resp) throws IOException
	{
		byte[] ret_b;
		long len;
		
		len = resp.getEntity().getContentLength();
		if(len == 0)
			return new byte[0];
		
		if(len < 0) {
			byte[] aux;
			InputStream in;
			int ret, count, pos;
			ArrayList<byte[]> buf_list;
			
			in = resp.getEntity().getContent();
			buf_list = new ArrayList<byte[]>();
			aux = new byte[4096];
			count = 0;
			
			for(int i = 0; i < this.max_content_len && (ret = in.read(aux)) > 0; i++) {
				byte[] buf_ele;
				
				buf_ele = new byte[ret];
			 
				System.arraycopy(aux, 0, buf_ele, 0, ret);
				buf_list.add(buf_ele);
				count += ret;
			}
			
			ret_b = new byte[count];
			pos = 0;
			while(buf_list.size() > 0) {
				byte[] buf_ele  = buf_list.remove(0);
				
				System.arraycopy(buf_ele, 0, ret_b, pos, buf_ele.length);
				pos += buf_ele.length;
			}
			
		} else {
			int ret_read;
			int i;
			InputStream in;
			
			in = resp.getEntity().getContent();			
			ret_b = new byte[(int)len];
			for(i = 0; i < len; ) {
				ret_read = in.read(ret_b, i, (int)len - i);
				if(ret_read <= 0)
					break;
				
				i += ret_read;
			}
			
			if(i != len)
				log.error("unexpected length => " + i + " - " + len);
		}
		
		return ret_b;
	}
	
	private Content getContentCached(SiteURL site) throws ContentEx, RecordEx
	{	
		ArrayList<Content> content_list;

		if(this.record == null)
			return null;
		
		content_list = (ArrayList<Content>)this.record.getObject("PROC_GetContentOnChannelHTTP", site.getURL().toString());
		if(content_list.size() == 0)
			return null;
	
		return content_list.get(0);
	}

	/* get content from a site url */
	public Content getContent(SiteURL site) throws ChannelEx
	{	
		byte[] content_b;
		String content_type;
		Content content_enc, content_ret;
		Header[] header_values;
		Header header;
		URL resp_url;
		Date last_modified;

		try { 
			Header locationHeader;
			int n_redirects;
			
			n_redirects = -1;
			
			do {
				n_redirects++;

				locationHeader = null;

				/* get content of database */
				content_ret = getContentCached(site);

				/* http response */
				this.http_resp = getResponse(content_ret, site);

				/* check status code */
				this.status_code = this.http_resp.getStatusLine().getStatusCode();

				/* get url */
				resp_url = this.http_req.getURI().toURL();


				switch(this.status_code) {
					case HttpStatus.SC_OK:
						break;

					/* redirect ? */
					case HttpStatus.SC_MOVED_TEMPORARILY:
					case HttpStatus.SC_MOVED_PERMANENTLY:

						/* save channel access */
						saveChannel(site, true);
						
						/* get location header for doing redirect */
						locationHeader = this.http_resp.getFirstHeader("Location");
						if (locationHeader != null) {
							String location;
							URL url_location;
							
							location = locationHeader.getValue();
							if(location.contains("://")) {
								url_location = new URL(location);
							} else {
								if(location.charAt(0) == '/')
									url_location = new URL(URLUtils.getURLHost(resp_url) + location);
								else
									url_location = new URL(URLUtils.getURLHost(resp_url) + "/" + location);
							}
								
							/* change url */
							site.setURL(url_location.toString());
						}
						

						break;

					case HttpStatus.SC_NOT_MODIFIED:
						if(content_ret != null) {
							this.cur_content = content_ret;
							return content_ret;
						}

						break;

					default:
						if(log.isDebugEnabled())
							log.warn(Thread.currentThread().getId() + " - " + this.status_code + " - url connected => " + resp_url.toString() );

						return null;
				}	

			} while(locationHeader != null && n_redirects < max_redirects);
			
			/* get content type */
			header_values = this.http_resp.getHeaders("Set-Cookie");
			for(int i = 0; i < header_values.length; i++)
				this.cookie_mgr.putCookie(resp_url, header_values[i].getValue());

			/* get content type */
			header = this.http_resp.getFirstHeader("Content-type");
			if(header == null)
				content_type = Content.MIMETYPE_OCTETBIN;
			else {
				int sep;
				
				content_type = header.getValue();
				sep = content_type.indexOf(';');
				if(sep > - 1)
					content_type = content_type.substring(0, sep);
			}

			if(log.isDebugEnabled()) {
				log.debug(this.status_code + " - url connected => " + resp_url.toString());				
				log.debug(this.status_code + " - original url => " + site.getURL().toString());
				log.debug(this.status_code + " - content type => " + content_type);
			}
			
			
			/* save cookies of server */
			header_values = this.http_resp.getHeaders("Set-Cookie");
			if(header_values != null) {
				for(int i = 0; i < header_values.length; i++)
					this.cookie_mgr.putCookie(this.http_req.getURI().toURL(), header_values[i].getValue());
			}
			
			/* get modified date */
			header = this.http_resp.getFirstHeader("Last-Modified");
			if(header == null)
				last_modified = null;
			else
				last_modified = DateUtil.parseDate(header.getValue());

			/* read content */
			content_b = readContent(this.http_resp);

			/* check if content is encoded */
			header = this.http_resp.getFirstHeader("Content-Encoding");
			if(header != null) {
				log.debug("content encoding => " + header.getValue() + " body length => " + content_b.length);				
				content_enc = ContentManager.getContent(header.getValue());
				if(content_enc == null) {
					log.warn("** i cant decode type (" + header.getValue() + ")");
					return null;
				}

				/* decode content */
				content_enc.setParentSite(site);				
				content_enc.parse(content_b);	

				content_b = content_enc.getBody();

			} else {	
				content_enc = null; 			
			}
			
			content_ret = ContentManager.getContent(content_type);

			/* redirect? */
			if(!site.getURL().toString().equals(resp_url.toString()))
				content_ret.setParentSite(new SiteURL(site.getURL()));
			else
				content_ret.setParentSite(site);
					
			if(last_modified != null)
				content_ret.setLastModified(last_modified);
			
			content_ret.setFormat(content_type);
			content_ret.setBody(content_b);
			
			this.cur_content = content_ret;
	
			if(log.isDebugEnabled()) 
				log.debug("content " + content_type + " body length => " + content_ret.getBody().length);				

		} catch(IOException io_ex) {
			throw new ChannelEx(io_ex);
			
		} catch(ContentEx cont_ex) {
			throw new ChannelEx(cont_ex);
			
		} catch(ChannelEx chan_ex) {
			throw chan_ex;
			
		} catch(Exception ex) {
			throw new ChannelEx(ex);	
			
		} finally {
			this.http_c.getConnectionManager().shutdown();
			this.http_c = null;
		}
		
		return content_ret;	
	}
	
	/* ignore some headers */
	boolean ignoreHeader(String header_name)
	{
		return header_name.equals("Connection") ||
			   header_name.startsWith("Accept-") ||
			   header_name.equals("Host") ||
			   header_name.equals("Set-Cookie")
			   ;
	}
	
	private void saveHeaders(int channel_id) throws ContentEx, RecordEx
	{
		Header h[];
		h = this.http_resp.getAllHeaders();
		if(h != null) {
			for(int i = 0; i < h.length; i++) {
				if(!ignoreHeader(h[i].getName()))
					this.record.execProc("PROC_CreateChannelHTTPHeader", channel_id, h[i].getName(), h[i].getValue());
			}
		}
	}
	
	private int saveChannel(SiteURL site , boolean do_changes) throws ContentEx, RecordEx
	{
		URL site_url;
		Header[] cookie_h;
		int port;
		int channel_id;
		
		site_url = site.getURL();
		
		port = site_url.getPort();
		if(port <= 0)
			port = site_url.getDefaultPort();
		
		cookie_h = this.http_req.getHeaders("Cookie");
		
		channel_id = (Integer)this.record.execProc("PROC_CreateChannelHTTP", this.spider.getSpiderId(), site_url.toString(), 
													   site_url.getHost(), port, 
													   cookie_h.length > 0? cookie_h[0].getValue(): null,
													   this.status_code
							);

		if(channel_id == -1)
			return -1;
		
		if(do_changes == true) {
			try { this.record.flush(); } catch(Exception ex) { throw new RecordEx(ex); };
		}
		
		/* save http headers */
		saveHeaders(channel_id);
			
		return channel_id;
	}
	
	public void saveContent() throws ContentEx, RecordEx
	{
		int channel_id;
		int ret_saved;
		
		if(this.cur_content == null)
			return;
		
		ret_saved = -1;
		
		try { 
			/* save channel */
			channel_id = saveChannel((SiteURL)this.cur_content.getParentSite(), false);
			
			/* save content */
			this.cur_content.save();
		
			/* link content and channel */
			ret_saved = (Integer)this.record.execProc("PROC_LinkChannelContent", channel_id, this.cur_content.getId(), "channelhttp", Utils.getObjectName(this.cur_content));			
				
		} finally {
			
			try {			
				if(ret_saved == -1)
					this.record.restore();
				else
					this.record.flush();
				
			} catch(Exception ex){ ex.printStackTrace(); }
			
		}
		
		
	}
		
	public Content getContent(Site site) throws ChannelEx
	{
		if(site instanceof SiteURL)
			return getContent((SiteURL)site);
		
		return null;
	}
}
